<!DOCTYPE html>
<html lang="en-us">
  <head>

    <meta http-equiv="content-type" content="text/html; charset=utf-8">
    
<meta charset="UTF-8">
<title>Collectors | Elasticsearch Guide [7.7] | Elastic</title>
<link rel="home" href="index.html" title="Elasticsearch Guide [7.7]">
<link rel="up" href="collecting-monitoring-data.html" title="Collecting monitoring data using legacy collectors">
<link rel="prev" href="collecting-monitoring-data.html" title="Collecting monitoring data using legacy collectors">
<link rel="next" href="es-monitoring-exporters.html" title="Exporters">
<meta name="DC.type" content="Learn/Docs/Elasticsearch/Reference/7.7">
<meta name="DC.subject" content="Elasticsearch">
<meta name="DC.identifier" content="7.7">
<meta name="robots" content="noindex,nofollow">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <script src="https://cdn.optimizely.com/js/18132920325.js"></script>
    <link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
    <link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
    <link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
    <link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
    <link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
    <link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
    <link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
    <link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
    <link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
    <link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
    <link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
    <link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
    <link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
    <link rel="manifest" href="/manifest.json">
    <meta name="apple-mobile-web-app-title" content="Elastic">
    <meta name="application-name" content="Elastic">
    <meta name="msapplication-TileColor" content="#ffffff">
    <meta name="msapplication-TileImage" content="/mstile-144x144.png">
    <meta name="theme-color" content="#ffffff">
    <meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd">
    <meta name="yandex-verification" content="d8a47e95d0972434">
    <meta name="localized" content="true">
    <meta name="st:robots" content="follow,index">
    <meta property="og:image" content="https://www.elastic.co/static/images/elastic-logo-200.png">
    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
    <link rel="icon" href="/favicon.ico" type="image/x-icon">
    <link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
    <link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
    <link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
    <!-- Give IE8 a fighting chance -->
    <!--[if lt IE 9]>
    <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
    <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
    <![endif]-->
    <link rel="stylesheet" type="text/css" href="/guide/static/styles.css">
  </head>

  <!--© 2015-2021 Elasticsearch B.V. Copying, publishing and/or distributing without written permission is strictly prohibited.-->

  <body>
    <!-- Google Tag Manager -->
    <script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
    <!-- End Google Tag Manager -->

    <!-- Global site tag (gtag.js) - Google Analytics -->
    <script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag(){dataLayer.push(arguments);}
      gtag('js', new Date());
      gtag('config', 'UA-12395217-16');
    </script>

    <!--BEGIN QUALTRICS WEBSITE FEEDBACK SNIPPET-->
    <script type="text/javascript">
      (function(){var g=function(e,h,f,g){
      this.get=function(a){for(var a=a+"=",c=document.cookie.split(";"),b=0,e=c.length;b<e;b++){for(var d=c[b];" "==d.charAt(0);)d=d.substring(1,d.length);if(0==d.indexOf(a))return d.substring(a.length,d.length)}return null};
      this.set=function(a,c){var b="",b=new Date;b.setTime(b.getTime()+6048E5);b="; expires="+b.toGMTString();document.cookie=a+"="+c+b+"; path=/; "};
      this.check=function(){var a=this.get(f);if(a)a=a.split(":");else if(100!=e)"v"==h&&(e=Math.random()>=e/100?0:100),a=[h,e,0],this.set(f,a.join(":"));else return!0;var c=a[1];if(100==c)return!0;switch(a[0]){case "v":return!1;case "r":return c=a[2]%Math.floor(100/c),a[2]++,this.set(f,a.join(":")),!c}return!0};
      this.go=function(){if(this.check()){var a=document.createElement("script");a.type="text/javascript";a.src=g;document.body&&document.body.appendChild(a)}};
      this.start=function(){var a=this;window.addEventListener?window.addEventListener("load",function(){a.go()},!1):window.attachEvent&&window.attachEvent("onload",function(){a.go()})}};
      try{(new g(100,"r","QSI_S_ZN_emkP0oSe9Qrn7kF","https://znemkp0ose9qrn7kf-elastic.siteintercept.qualtrics.com/WRSiteInterceptEngine/?Q_ZID=ZN_emkP0oSe9Qrn7kF")).start()}catch(i){}})();
    </script><div id="ZN_emkP0oSe9Qrn7kF"><!--DO NOT REMOVE-CONTENTS PLACED HERE--></div>
    <!--END WEBSITE FEEDBACK SNIPPET-->

    <div id="elastic-nav" style="display:none;"></div>
    <script src="https://www.elastic.co/elastic-nav.js"></script>

    <!-- Subnav -->
    <div>
      <div>
        <div class="tertiary-nav d-none d-md-block">
          <div class="container">
            <div class="p-t-b-15 d-flex justify-content-between nav-container">
              <div class="breadcrum-wrapper"><span><a href="/guide/" style="font-size: 14px; font-weight: 600; color: #000;">Docs</a></span></div>
            </div>
          </div>
        </div>
      </div>
    </div>

    <div class="main-container">
      <section id="content">
        <div class="content-wrapper">

          <section id="guide" lang="en">
            <div class="container">
              <div class="row">
                <div class="col-xs-12 col-sm-8 col-md-8 guide-section">
                  <!-- start body -->
                  <div class="page_header">
<strong>IMPORTANT</strong>: No additional bug fixes or documentation updates
will be released for this version. For the latest information, see the
<a href="../current/index.html">current release documentation</a>.
</div>
<div id="content">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="index.html">Elasticsearch Guide [7.7]</a></span>
»
<span class="breadcrumb-link"><a href="monitor-elasticsearch-cluster.html">Monitor a cluster</a></span>
»
<span class="breadcrumb-link"><a href="collecting-monitoring-data.html">Collecting monitoring data using legacy collectors</a></span>
»
<span class="breadcrumb-node">Collectors</span>
</div>
<div class="navheader">
<span class="prev">
<a href="collecting-monitoring-data.html">« Collecting monitoring data using legacy collectors</a>
</span>
<span class="next">
<a href="es-monitoring-exporters.html">Exporters »</a>
</span>
</div>
<div class="section xpack">
<div class="titlepage"><div><div>
<h2 class="title">
<a id="es-monitoring-collectors"></a>Collectors<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/monitoring/collectors.asciidoc">edit</a><a class="xpack_tag" href="/subscriptions"></a>
</h2>
</div></div></div>
<div class="important admon">
<div class="icon"></div>
<div class="admon_content">
<p>Metricbeat is the recommended method for collecting and shipping monitoring
data to a monitoring cluster.</p>
<p>If you have previously configured legacy collection methods, you should migrate
to using Metricbeat collection methods. Use either Metricbeat collection or
legacy collection methods; do not use both.</p>
<p>Learn more about <a class="xref" href="configuring-metricbeat.html" title="Collecting Elasticsearch monitoring data with Metricbeat"><em>Collecting monitoring data with Metricbeat</em></a>.</p>
</div>
</div>
<p>Collectors, as their name implies, collect things. Each collector runs once for
each collection interval to obtain data from the public APIs in Elasticsearch and X-Pack
that it chooses to monitor. When the data collection is finished, the data is
handed in bulk to the <a class="xref" href="es-monitoring-exporters.html" title="Exporters">exporters</a> to be sent to the
monitoring clusters. Regardless of the number of exporters, each collector only
runs once per collection interval.</p>
<p>There is only one collector per data type gathered. In other words, for any
monitoring document that is created, it comes from a single collector rather
than being merged from multiple collectors. The Elasticsearch monitoring features
currently have a few collectors because the goal is to minimize overlap between
them for optimal performance.</p>
<p>Each collector can create zero or more monitoring documents. For example,
the <code class="literal">index_stats</code> collector collects all index statistics at the same time to
avoid many unnecessary calls.</p>
<div class="informaltable">
<table border="1" cellpadding="4px">
<colgroup>
<col class="col_1">
<col class="col_2">
<col class="col_3">
</colgroup>
<thead>
<tr>
<th align="left" valign="top">Collector</th>
<th align="left" valign="top">Data Types</th>
<th align="left" valign="top">Description</th>
</tr>
</thead>
<tbody>
<tr>
<td align="left" valign="top"><p>Cluster Stats</p></td>
<td align="left" valign="top"><p><code class="literal">cluster_stats</code></p></td>
<td align="left" valign="top"><p>Gathers details about the cluster state, including parts of the actual cluster
state (for example <code class="literal">GET /_cluster/state</code>) and statistics about it (for example,
<code class="literal">GET /_cluster/stats</code>). This produces a single document type. In versions prior
to X-Pack 5.5, this was actually three separate collectors that resulted in
three separate types: <code class="literal">cluster_stats</code>, <code class="literal">cluster_state</code>, and <code class="literal">cluster_info</code>. In
5.5 and later, all three are combined into <code class="literal">cluster_stats</code>. This only runs on
the <em>elected</em> master node and the data collected (<code class="literal">cluster_stats</code>) largely
controls the UI. When this data is not present, it indicates either a
misconfiguration on the elected master node, timeouts related to the collection
of the data, or issues with storing the data. Only a single document is produced
per collection.</p></td>
</tr>
<tr>
<td align="left" valign="top"><p>Index Stats</p></td>
<td align="left" valign="top"><p><code class="literal">indices_stats</code>, <code class="literal">index_stats</code></p></td>
<td align="left" valign="top"><p>Gathers details about the indices in the cluster, both in summary and
individually. This creates many documents that represent parts of the index
statistics output (for example, <code class="literal">GET /_stats</code>). This information only needs to
be collected once, so it is collected on the <em>elected</em> master node. The most
common failure for this collector relates to an extreme number of indices — and
therefore time to gather them — resulting in timeouts. One summary
<code class="literal">indices_stats</code> document is produced per collection and one <code class="literal">index_stats</code>
document is produced per index, per collection.</p></td>
</tr>
<tr>
<td align="left" valign="top"><p>Index Recovery</p></td>
<td align="left" valign="top"><p><code class="literal">index_recovery</code></p></td>
<td align="left" valign="top"><p>Gathers details about index recovery in the cluster. Index recovery represents
the assignment of <em>shards</em> at the cluster level. If an index is not recovered,
it is not usable. This also corresponds to shard restoration via snapshots. This
information only needs to be collected once, so it is collected on the <em>elected</em>
master node. The most common failure for this collector relates to an extreme
number of shards — and therefore time to gather them — resulting in timeouts.
This creates a single document that contains all recoveries by default, which
can be quite large, but it gives the most accurate picture of recovery in the
production cluster.</p></td>
</tr>
<tr>
<td align="left" valign="top"><p>Shards</p></td>
<td align="left" valign="top"><p><code class="literal">shards</code></p></td>
<td align="left" valign="top"><p>Gathers details about all <em>allocated</em> shards for all indices, particularly
including what node the shard is allocated to. This information only needs to be
collected once, so it is collected on the <em>elected</em> master node. The collector
uses the local cluster state to get the routing table without any network
timeout issues unlike most other collectors. Each shard is represented by a
separate monitoring document.</p></td>
</tr>
<tr>
<td align="left" valign="top"><p>Jobs</p></td>
<td align="left" valign="top"><p><code class="literal">job_stats</code></p></td>
<td align="left" valign="top"><p>Gathers details about all machine learning job statistics (for example, <code class="literal">GET
/_ml/anomaly_detectors/_stats</code>). This information only needs to be collected
once, so it is collected on the <em>elected</em> master node. However, for the master
node to be able to perform the collection, the master node must have
<code class="literal">xpack.ml.enabled</code> set to true (default) and a license level that supports machine learning.</p></td>
</tr>
<tr>
<td align="left" valign="top"><p>Node Stats</p></td>
<td align="left" valign="top"><p><code class="literal">node_stats</code></p></td>
<td align="left" valign="top"><p>Gathers details about the running node, such as memory utilization and CPU
usage (for example, <code class="literal">GET /_nodes/_local/stats</code>). This runs on <em>every</em> node with
monitoring features enabled. One common failure results in the timeout of the node
stats request due to too many segment files. As a result, the collector spends
too much time waiting for the file system stats to be calculated until it
finally times out. A single <code class="literal">node_stats</code> document is created per collection.
This is collected per node to help to discover issues with nodes communicating
with each other, but not with the monitoring cluster (for example, intermittent
network issues or memory pressure).</p></td>
</tr>
</tbody>
</table>
</div>
<p>The Elasticsearch monitoring features use a single threaded scheduler to run the
collection of Elasticsearch monitoring data by all of the appropriate collectors on each
node. This scheduler is managed locally by each node and its interval is
controlled by specifying the <code class="literal">xpack.monitoring.collection.interval</code>, which
defaults to 10 seconds (<code class="literal">10s</code>), at either the node or cluster level.</p>
<p>Fundamentally, each collector works on the same principle. Per collection
interval, each collector is checked to see whether it should run and then the
appropriate collectors run. The failure of an individual collector does not
impact any other collector.</p>
<p>Once collection has completed, all of the monitoring data is passed to the
exporters to route the monitoring data to the monitoring clusters.</p>
<p>If gaps exist in the monitoring charts in Kibana, it is typically because either
a collector failed or the monitoring cluster did not receive the data (for
example, it was being restarted). In the event that a collector fails, a logged
error should exist on the node that attempted to perform the collection.</p>
<div class="note admon">
<div class="icon"></div>
<div class="admon_content">
<p>Collection is currently done serially, rather than in parallel, to avoid
      extra overhead on the elected master node. The downside to this approach
      is that collectors might observe a different version of the cluster state
      within the same collection period. In practice, this does not make a
      significant difference and running the collectors in parallel would not
      prevent such a possibility.</p>
</div>
</div>
<p>For more information about the configuration options for the collectors, see
<a class="xref" href="monitoring-settings.html#monitoring-collection-settings" title="Monitoring Collection Settings">Monitoring Collection Settings</a>.</p>
<h5>
<a id="es-monitoring-stack"></a>Collecting data from across the Elastic Stack<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/monitoring/collectors.asciidoc">edit</a>
</h5>
<p>Elasticsearch monitoring features also receive monitoring data from other parts of the
Elastic Stack. In this way, it serves as an unscheduled monitoring data
collector for the stack.</p>
<p>By default, data collection is disabled. Elasticsearch monitoring data is not
collected and all monitoring data from other sources such as Kibana, Beats, and
Logstash is ignored. You must set <code class="literal">xpack.monitoring.collection.enabled</code> to <code class="literal">true</code>
to enable the collection of monitoring data. See <a class="xref" href="monitoring-settings.html" title="Monitoring settings in Elasticsearch">Monitoring settings</a>.</p>
<p>Once data is received, it is forwarded to the exporters
to be routed to the monitoring cluster like all monitoring data.</p>
<div class="warning admon">
<div class="icon"></div>
<div class="admon_content">
<p>Because this stack-level "collector" lives outside of the collection
interval of Elasticsearch monitoring features, it is not impacted by the
<code class="literal">xpack.monitoring.collection.interval</code> setting. Therefore, data is passed to the
exporters whenever it is received. This behavior can result in indices for Kibana,
Logstash, or Beats being created somewhat unexpectedly.</p>
</div>
</div>
<p>While the monitoring data is collected and processed, some production cluster
metadata is added to incoming documents. This metadata enables Kibana to link the
monitoring data to the appropriate cluster. If this linkage is unimportant to
the infrastructure that you’re monitoring, it might be simpler to configure
Logstash and Beats to report monitoring data directly to the monitoring cluster.
This scenario also prevents the production cluster from adding extra overhead
related to monitoring data, which can be very useful when there are a large
number of Logstash nodes or Beats.</p>
<p>For more information about typical monitoring architectures, see
<a class="xref" href="how-monitoring-works.html" title="How monitoring works"><em>How it works</em></a>.</p>
</div>
<div class="navfooter">
<span class="prev">
<a href="collecting-monitoring-data.html">« Collecting monitoring data using legacy collectors</a>
</span>
<span class="next">
<a href="es-monitoring-exporters.html">Exporters »</a>
</span>
</div>
</div>

                  <!-- end body -->
                </div>
                <div class="col-xs-12 col-sm-4 col-md-4" id="right_col">
                  <div id="rtpcontainer" style="display: block;">
                    <div class="mktg-promo">
                      <h3>Most Popular</h3>
                      <ul class="icons">
                        <li class="icon-elasticsearch-white"><a href="https://www.elastic.co/webinars/getting-started-elasticsearch?baymax=default&amp;elektra=docs&amp;storm=top-video">Get Started with Elasticsearch: Video</a></li>
                        <li class="icon-kibana-white"><a href="https://www.elastic.co/webinars/getting-started-kibana?baymax=default&amp;elektra=docs&amp;storm=top-video">Intro to Kibana: Video</a></li>
                        <li class="icon-logstash-white"><a href="https://www.elastic.co/webinars/introduction-elk-stack?baymax=default&amp;elektra=docs&amp;storm=top-video">ELK for Logs &amp; Metrics: Video</a></li>
                      </ul>
                    </div>
                  </div>
                </div>
              </div>
            </div>
          </section>

        </div>


<div id="elastic-footer"></div>
<script src="https://www.elastic.co/elastic-footer.js"></script>
<!-- Footer Section end-->

      </section>
    </div>

<script src="/guide/static/jquery.js"></script>
<script type="text/javascript" src="/guide/static/docs.js"></script>
<script type="text/javascript">
  window.initial_state = {}</script>
  </body>
</html>
